package net.trustie.webmagic.one;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import javax.annotation.Resource;

import net.trustie.webmagic.extrator.CSSPathExtractor;
import net.trustie.webmagic.extrator.Extractor;
import net.trustie.webmagic.extrator.URLPatternExtractor;
import net.trustie.webmagic.one.dao.DetailHtmlDao;
import net.trustie.webmagic.one.dao.ErrorPageDao;
import net.trustie.webmagic.one.model.DetailHtml;
import net.trustie.webmagic.utils.LongSleepHelper;
import net.trustie.webmagic.utils.MyRandomer;
import net.trustie.webmagic.utils.SubSite;

import org.apache.log4j.Logger;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.utils.UrlUtils;

@Component
public class Crawler {
	private HttpClientDownloader downloader = new HttpClientDownloader();
	private Configure conf;
	private Set<Integer> acceptStatCode;
	private String domain;
	private Site site;
	private MyTask task = new MyTask();
	Logger logger = Logger.getLogger(this.getClass());
	Map<String,Set<String>> lastPages = new HashMap<String,Set<String>>();
	
	@Resource
	private DetailHtmlDao detailHtmlDao;

	@Resource
	private ErrorPageDao errorPageDao;
	private String detailHtmlTableName;
	private String errorPageTableName;
	
	public static void main(String args[]){
		String configureName = "";
		if (args.length != 0)
			configureName = args[0].toString();
		if (configureName.equals("")) {
			configureName = "ossean";
		} else {
		}
		ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
				"classpath:/spring/applicationContext*.xml");
		final Crawler htmlCrawler = applicationContext
				.getBean(Crawler.class);
		htmlCrawler.crawl(configureName);
		System.out.println("this is in th crawler");
	}

	
	private void crawl(String configureName) {
		// TODO Auto-generated method stub
		init(configureName);
		
		Extractor extractor = null;
		if ("URLPattern".equals(conf.getExtractMethod())) {
			extractor = new URLPatternExtractor(conf.getUrlPost());
		} else if ("CSSPath".equals(conf.getExtractMethod())) {
			extractor = new CSSPathExtractor(conf.getUrlPost());
		} else {
			logger.error("extract method error!!!");
		}
		
		int subSiteNum = this.conf.getSubSiteNum();
		logger.info("该站点共有" + subSiteNum + "个子站点");	
		
		SubSite subSite;
		
		while(true){
			int doneNum = 0;//记录已完成镜像爬取的子站点个数
			while(doneNum++ < subSiteNum){
				//conf 负责获得这次处理的站点
				subSite = this.conf.nexSite();
				
				logger.info("现在爬取子站点  " + subSite.getName());
				
				Page page = downloader.download(new Request(
						subSite.getPrefixUrl()
						+ subSite.getStartPageIndex()
						+ subSite.getPostfixUrl()), task);
				
				Html html = new Html(UrlUtils.fixAllRelativeHrefs(
						page.getRawText(),this.conf.getFixAllRelativeHrefs()));

				Set<String> nowUrls = extractor.extract(html);
				String pageName = this.domain + subSite.getName();
				Set<String> lastUrls = lastPages.get(pageName);
				
				Set<String> newUrls = new HashSet<String>();
				if(lastUrls == null){
					logger.info("首爬 - 全新");
					lastPages.put(pageName,nowUrls);
					newUrls = nowUrls;
				}else{
					for(String url : nowUrls){
						if(!lastUrls.contains(url)){
							newUrls.add(url);
							logger.info("new url : " + url);
						}
					}
				}
				lastPages.put(pageName,nowUrls);
				if(newUrls.isEmpty()){
					logger.info("no new urls");
				}
				
				
				for(String url : newUrls){
					page = downloader.download(new Request(url), task);
					DetailHtml detailHtml = new DetailHtml();
					detailHtml.setStatusCode(page.getStatusCode());
					detailHtml.setHtml(page.getRawText());
					detailHtml.setUrl(page.getRequest().getUrl());
					if (page.getStatusCode() == 200) {
						detailHtmlDao.insertDetailHtml(detailHtml ,
								this.detailHtmlTableName);
						logger.info("save page " + page.getUrl() + " success");
					} else {
						errorPageDao.insertPage(this.errorPageTableName,
								detailHtml);
						logger.info("save error page " + detailHtml.getUrl() + "error"
								+ detailHtml.getStatusCode() + " success!");
					}
					int time = MyRandomer.getRandomNum(conf.getMinInterval(),
							conf.getMaxInterval());
					try {
						logger.info("sleep " + time / 1000 + " s ...");
						Thread.sleep(time);
						
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}
			}
			try {
				int time = MyRandomer.getRandomNum(conf.getMinlongSleepTime(),conf.getMaxlongSleepTime());
				logger.info("next round will start in " + time / 1000 + " s...");
				Thread.sleep(time);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		
		
		
	}


	private void init(String configureName) {
		// TODO Auto-generated method stub
		this.conf = new Configure(configureName);
		this.acceptStatCode = this.initAcceptStatCode();
		this.domain = conf.getDomain();
		this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
				.setTimeOut(conf.getTimeOut())
				.setRetryTimes(conf.getRetryTimes())
				.setCycleRetryTimes(conf.getCycleRetryTimes())
				.setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
				.setAcceptStatCode(acceptStatCode);
		task.site = site;
		
		this.detailHtmlTableName = this.domain + "_html_detail";
		this.errorPageTableName = this.domain + "_error_page";
	}


	private Set<Integer> initAcceptStatCode() {
		Set<Integer> acceptStatCode = new HashSet<Integer>();
		acceptStatCode.add(200);
		acceptStatCode.add(404);
		acceptStatCode.add(500);
		acceptStatCode.add(503);
		return acceptStatCode;
	}
}

class MyTask implements Task{
	Site site;
	@Override
	public String getUUID() {
		// TODO Auto-generated method stub
		return null;
	}

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	@Override
	public int getThreadAlive() {
		// TODO Auto-generated method stub
		return 0;
	}
}